const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  const browser = await puppeteer.launch({headless: false, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  let totalBlogs = [];
  const SLEEP_TIME = 200;
  const tab = 'segmentfault';
  const url = `https://k.sina.cn/media_5454994512.html`;

  try {
    await page.goto(url, {waitUntil: 'domcontentloaded'});
    await page.waitFor(1000);
  } catch (e) {
    this.ctx.logger.error(`fetch ${tab}-${subTab} failure, reason is : ${e.message}`);
    return totalBlogs;
  }

  let preScrollHeight = 0;
  let scrollHeight = -1;
  let cnt = 0;
  let retry_cnt = 0;
  while (cnt < 30) {
    console.log(cnt);
    // 详情信息是根据滚动异步加载，所以需要让页面滚动到屏幕最下方，通过延迟等待的方式进行多次滚动
    let scrollH1 = await page.evaluate(async () => {
      let h1 = document.body.scrollHeight;
      window.scrollTo(0, h1);
      return h1;
    });

    // 延迟等待时间
    await page.waitFor(SLEEP_TIME);

    let scrollH2 = await page.evaluate(async () => {
      return document.body.scrollHeight;
    });
    let scrollResult = [scrollH1, scrollH2];
    preScrollHeight = scrollResult[0];
    scrollHeight = scrollResult[1];

    if (preScrollHeight == scrollHeight) {
      retry_cnt++;
      if (retry_cnt > 20) {
        break;
      }
      try {
        await page.click('#btn-load-more > a');
      } catch (e) {

      }
    } else {
      retry_cnt = 0;
      cnt++;
    }
  }

  totalBlogs = await page.$$eval('#j_oWrap > dl', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('dd > a > h4').innerText;
      let content = ele.querySelector('dd > a').href;
      tmpBlogs.push({
        title: title,
        content: content,
        original_url: content,
        tab: "other",
        sub_tab: "搞笑",
        source: '囧哥说事'
      });
    });
    return tmpBlogs;
  });


  let writerStream = fs.createWriteStream(`crawler_data/jiongge.json`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

};


main();


